package spider;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import exception.CodeException;
import org.apache.http.HttpStatus;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import util.AgentUtil;

import java.io.IOException;
import java.util.Optional;

/**
 * @author zhanglinfeng
 * @date create in 2022/10/11 15:40
 */
public class MyPipeline {

    public Document getDocument(String url) throws IOException, InterruptedException {
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        webClient.getOptions().setJavaScriptEnabled(true);              // 启用JS解释器，默认为true
        webClient.getOptions().setCssEnabled(false);                    // 禁用css支持
        webClient.getOptions().setThrowExceptionOnScriptError(false);   // js运行错误时，是否抛出异常
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
        webClient.getOptions().setTimeout(8000);                  // 设置连接超时时间
        webClient.getOptions().setActiveXNative(false);//不启用ActiveX
        webClient.getOptions().setCssEnabled(false);//不展示页面，不启用css
        webClient.getOptions().setDownloadImages(false);//不下载图片
        webClient.getOptions().setUseInsecureSSL(true);
        webClient.getCookieManager().setCookiesEnabled(true);
        webClient.setAjaxController(new NicelyResynchronizingAjaxController());//支持ajax
        webClient.setJavaScriptErrorListener(new MyJavaScriptErrorListener()); //不打印JS错误
        webClient.addRequestHeader("User-Agent", AgentUtil.get());
        HtmlPage htmlPage = webClient.getPage(url);
        Thread.sleep(3000);
        webClient.waitForBackgroundJavaScript(7000);
        int statusCode = htmlPage.getWebResponse().getStatusCode();
        if (HttpStatus.SC_OK != statusCode) {
            throw new CodeException("statusCode error", String.valueOf(statusCode));
        }
        webClient.close();
        return Optional.ofNullable(htmlPage).map(HtmlPage::asXml).map(Jsoup::parse).orElse(null);
    }

}
